package blod.spider;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;
import us.codecraft.webmagic.scheduler.Scheduler;
import us.codecraft.webmagic.selector.Html;

import java.util.ArrayList;
import java.util.List;

/**
 * 1、实现代理 ?
 */
public class Start implements PageProcessor {
    private Site site = Site.me().setCycleRetryTimes(2).setRetryTimes(2).setSleepTime(3000).setTimeOut(3 * 60 * 1000)
            .setUserAgent(UserAgent.get())
            .addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
            .addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
            .setCharset("UTF-8");

    @Override
    public void process(Page page) {
        Html html = page.getHtml();
        System.out.println(html.toString());
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        List<String> list = new ArrayList<>();
        list.add("https://search.jd.com/Search?keyword=%E6%99%BA%E8%83%BD%E6%9C%BA&enc=utf-8&wq=%E6%99%BA%E8%83%BD%E6%9C%BA&pvid=894163e9e6c241e69f0b2b5b6d8e13d8");
        /*DBUtil.init();*/
        Spider.create(new Start())
                //.addPipeline(new ConsolePipeline())
                //.addPipeline(new FilePipeline(System.getProperty("user.dir") + "/logs/record/"))
                .addUrl(list.toArray(new String[list.size()]))
              //  .setScheduler(new FileCacheQueueScheduler(System.getProperty("user.dir") + "/logs/test/"))
                .thread(3)
                .run();
    }
}
